#!/usr/local/bin/python
import httplib
import urllib
import json
import sys
import os
import Queue
import threading
import random
import uuid
import datetime
import chardet

#the first must be rowkey id
#the follows must match the fields of each line in datafile
tianya_user_login_log_fields = 'rkey_id,userid,username,client_ip,data_time'
tianya_user_login_log_store = 'hbase'
tianya_user_login_log_address = '127.0.0.1'
tianya_user_login_log_desc = 'tianay_user_login_log'

tianya_user_login_log_test_fields = 'rkey_id,userid,username,client_ip,data_time'
tianya_user_login_log_test_store = 'hbase'
tianya_user_login_log_test_address = '127.0.0.1'
tianya_user_login_log_test_desc = 'tianay_user_login_test_log'

webhost = '19.2.174.82'
#webhost = '127.0.0.1'
q = Queue.Queue(maxsize = 1000)

def create(*args):
    table = args[0].strip()
    try:
        str = '%s_fields' % (table, )
        fields = globals()[str]
        str = '%s_store' % (table, )
        store = globals()[str]
        str = '%s_address' % (table, )
        address = globals()[str]
        str = '%s_desc' % (table, )
        desc = globals()[str]
    except KeyError:
        print 'please specify %s schema' % (table,)
        sys.exit(-1)

    str = fields.split(',')
    rowkey = str[0]

    if (len(table) == 0 or
        len(rowkey) == 0 or
        len(fields) == 0 or
        len(store) == 0 or
        len(address) == 0 or
        len(desc) == 0):
        print 'name, rowkey, fields, store, address, desc must not empty\n'
        sys.exit(-1);
    url = '/%s/create?store=%s&field=%s&rowkey=%s&desc=%s&address=%s' % (table, store, fields, rowkey, desc, address)
    #print url
    conn = httplib.HTTPConnection(webhost, 80)
    conn.request('GET', url)
    res = conn.getresponse()
    if (res.status != 200):
        print 'create <%s> error: status[ %d ] emsg[ %s ]\n' % (table, res.status, res.read())
    else:
        print 'create <%s> ok\n' % (table, )
    conn.close()

def jobs(job, table, f_log, thread_id):
    while (True):
        data = q.get()
        q.task_done()
        #f_log.write("%d running\n" % (thread_id,))
        if (data == 'good game'):
            break
        if (len(data) != 0):
            job(table, data, f_log)
        else:
            f_log.write("from queue, empty data\n")
            f_log.flush()
    f_log.write('thread %d done\n' % (thread_id,))
    f_log.flush()

def tianya_user_login_log_add(table, data, f_log):
    jsonhead = {}
    jsonstr = []
    url = '/%s/add' % (table,)
    fields = globals()['%s_fields' % (table, )].split(',')
    lines = data.split('\r\n')
    line_num = len(lines)
    for j in range(0, line_num):
        line = lines[j]
        if (len(line) != 0):
            _fields = line.split('\x04')
            if (len(_fields) != len(fields)):
                f_log.write('(%s) != (%s), ignore the line\n' % (_fields, fields))
                f_log.flush()
                continue
            jstr = {}
            jstr[fields[0]] = _fields[4].split()[0] + '-' + str(uuid.uuid4())
            for i in range(1, len(_fields)):
                jstr[fields[i]] = _fields[i]
            jsonstr.append(jstr)
        else:
            if (j < line_num-1):
                f_log.write("null line, ignore: %s\n" % (data,));
                f_log.flush()
    if (len(jsonstr) == 0):
        f_log.write("jsonstr empty: %s\n" % (data,))
        f_log.flush()
        return
    jsonhead["array"] = jsonstr
    try:
        jsonstr = json.dumps(jsonhead, encoding='utf8', ensure_ascii=False).encode('utf8')
    except:
        f_log.write('json error: %s\n' % (data,))
        return
    #f_log.write('%s\n' % (jsonstr,))
    #return

    conn = ''
    try:
        conn = httplib.HTTPConnection(webhost, 80)
        #conn.debuglevel = 1
        conn.connect()
    except:
        f_log.write('TABLE=>%s: CONNECT TO WEB SERVER ERROR\n%s\n' % (table, data))
        return
    conn.putrequest("POST", url)
    conn.putheader("Content-Length", len(jsonstr))
    conn.putheader("Content-Type", "text/json")
    #conn.putheader("Connection", "close")
    conn.endheaders()

    try:
        conn.send(jsonstr)
    except:
        f_log.write('TABLE=>%s: SEND DATA TO WEB SERVER ERROR\n%s\n' % (table, data))
        conn.close()
        return

    res = ''
    try:
        res = conn.getresponse()
    except:
        f_log.write('TABLE=>%s: GET RESPONSE FROM WEB SERVER ERROR\n%s\n' % (table, data))
        conn.close()
        return
    if (res.status != 200):
        f_log.write('TABLE=>%s:\nDATASERVER RETURN CODE=>%d\nDATASERVER RETURN REASON=>%s\nDATASERVER RETURN CONTENT=>%s\n' % (table, res.status, res.reason, res.read()))
        f_log.write('LOCAL CONTENT=>%s\n' % (data,))
        f_log.flush()
    conn.close()

def tianya_user_info_add(table, data, f_log):
    #jsonhead = '{"array":['
    jsonhead = {}
    #jsonstr = ''
    jsonstr = []
    #jsontail = ']}'
    url = '/%s/add' % (table,)
    fields = globals()['%s_fields' % (table, )].split(',')
    lines = data.split('\r\n')
    line_num = len(lines)
    for j in range(0, line_num):
        line = lines[j]
        if (len(line) != 0):
            _fields = line.split('\x04')
            if (len(_fields) != len(fields)):
                f_log.write('(%s) != (%s), ignore the line\n' % (_fields, fields))
                continue
            #jsonstr += '{'
            jstr = {}
            for i in range(0, len(_fields)):
                #_fields[i] = _fields[i].replace("\\", "\\\\")
                #_fields[i] = _fields[i].replace('"', '\\"')
                #jsonstr += '"%s":"%s"' % (fields[i], _fields[i])
                jstr[fields[i]] = _fields[i]
                #if (i != len(_fields)-1):
                    #jsonstr += ','
            #jsonstr += '},'
            jsonstr.append(jstr)
        else:
            if (j < line_num-1):
                f_log.write("null line, ignore: %s\n" % (data,));
                f_log.flush()
    if (len(jsonstr) == 0):
        f_log.write("jsonstr empty: %s\n" % (data,))
        f_log.flush()
        return
    jsonhead["array"] = jsonstr
    #jsonstr = jsonstr[0:len(jsonstr)-1]
    #jsonstr = jsonhead + jsonstr + jsontail
    try:
        jsonstr = json.dumps(jsonhead, encoding='utf8',  ensure_ascii=False).encode('utf8')
    except:
        f_log.write('json error: %s\n' % (data,))
        return
    #f_log.write('%s\n' % (jsonstr,))
    #return

    conn = ''
    try:
        conn = httplib.HTTPConnection(webhost, 80)
        #conn.debuglevel = 1
        conn.connect()
    except:
        f_log.write('TABLE=>%s: CONNECT TO WEB SERVER ERROR\n%s\n' % (table, data))
        return
    conn.putrequest("POST", url)
    conn.putheader("Content-Length", len(jsonstr))
    conn.putheader("Content-Type", "text/json")
    #conn.putheader("Connection", "close")
    conn.endheaders()

    try:
        conn.send(jsonstr)
    except:
        f_log.write('TABLE=>%s: SEND DATA TO WEB SERVER ERROR\n%s\n' % (table, data))
        conn.close()
        return

    res = ''
    try:
        res = conn.getresponse()
    except:
        f_log.write('TABLE=>%s: GET RESPONSE FROM WEB SERVER ERROR\n%s\n' % (table, data))
        conn.close()
        return
    if (res.status != 200):
        f_log.write('TABLE=>%s:\nDATASERVER RETURN CODE=>%d\nDATASERVER RETURN REASON=>%s\nDATASERVER RETURN CONTENT=>%s\n' % (table, res.status, res.reason, res.read()))
        f_log.write('LOCAL CONTENT=>%s\n' % (data,))
        f_log.flush()
    conn.close()

def add(*args):
    if (len(args) < 2):
        print 'action "add" need more arguments'
        sys.exit(-1);
    #os.chdir('/www/userdata/tianya_data');
    table = args[0].strip()
    if (len(table) == 0):
        print 'action "add" must specify tablename.'
        sys.exit(-1)
    args = args[1:]
    for i in range(0, len(args)):
        datafile = args[i].strip()
        print '%s' % (datafile,)

        if (len(datafile) == 0):
            print 'action "add" must specify datafile.'
            continue

        if (datafile[0] != '/'):
            print 'action "add" <%s> must be absolute path.' % (datafile,)
            continue

        multithread = False

        #print table
        #print datafile

        if (os.path.isfile(datafile) == False):
            print 'action "add" datafile must exist' % (datafile,)
            continue
            #sys.exit(-1)

        if (os.path.getsize(datafile) > 10*1024*1024):
            multithread = True

        count = 0
        f_log = open('logs/%s_add.log.%s' % (table, os.path.basename(datafile)), "wt")
        try:
            job = globals()['%s_add' % (table, )]
        except KeyError:
            f_log.write("%s invalid\n" % (table, ))
            continue

        f = open(datafile, "rt")
        startime = datetime.datetime.now()
        if (multithread):
            threads = []
            for i in range(0, 5):
                threads.append(threading.Thread(target=jobs, args=(job, table, f_log, i)));
                threads[i].start()
            while (True):
                data = ''
                for i in range(0, 100):
                    line = f.readline()
                    if not line:
                        break
                    count += 1
                    data += line
                if (len(data) == 0):
                    q.put("good game");
                    q.put("good game");
                    q.put("good game");
                    q.put("good game");
                    q.put("good game");
                    break;
                q.put(data)

            q.join()
            for t in threads:
                t.join()
        else:
            while (True):
                data = ''
                for i in range(0, 1000):
                    line = f.readline()
                    if not line:
                        break
                    count += 1
                    data += line
                if (len(data) == 0):
                    break;
                job(table, data, f_log)

        f.close()
        endtime = datetime.datetime.now()
        totaltime = (endtime-startime).seconds
        hour = totaltime / 3600
        mins = (totaltime % 3600) / 60
        secds = (totaltime % 3600) % 60
        f_log.write('time used: %dH %dM %dS\n' % (hour, mins, secds))
        if (totaltime != 0):
            f_log.write('qps: %d\n' % (count/totaltime, ))
            f_log.flush()
        else:
            f_log.write('count: %d\n' % (count,))
            f_log.flush()
        f_log.close()

if __name__ == '__main__':
    if len(sys.argv) < 2:
        print 'Usage arguments: action args\n'
        sys.exit(-1)
    args = []
    action = sys.argv[1]
    args.extend(sys.argv[2:])
    func = globals()[action];
    func(*args);
